#!/bin/sh
#
# This script runs all test scripts that use the unittest library
#
# unittest provides functions that quit NEST with a non-zero exit
# code upon errors. This is checked here and used to print a nice
# summary after all tests have been run.
#
# For testing this script, these files may be handy:
#
#   exitcode0.sli
#   exitcode1.sli
#   exitcode2.sli
#   exitcode3.sli
#   exitcode99.sli
#   exitcode126.sli
#
# They return a specific exit code
#

#
# usage [exit_code bad_option]
#
usage ()
{
    if test $1 -ne 0 ; then
        echo "Unknown option: $2"
    fi

    cat <<EOF
Usage: do_tests.sh [options ...]"

Options:

    --help              Print program options and exit
    --test-pynest       Test the PyNEST installation and APIs
    --output-dir=/path  Output directory (default: ./reports)
EOF

    exit $1
}

TEST_PYNEST=false

while test $# -gt 0 ; do
    case "$1" in
        --help)
            usage 0
            ;;
        --test-pynest)
            TEST_PYNEST=true
            ;;
        --output-dir=*)
            TEST_OUTDIR="$( echo "$1" | sed 's/^--output-dir=//' )"
            ;;
        *)
            usage 1 "$1"
            ;;
    esac
    shift
done

#
# bail_out message
#
bail_out ()
{
    echo "$1"
    exit 1
}

#
# ask_results
#
ask_results ()
{
    echo "***"
    echo "*** Please report the problem at"
    echo "***     https://github.com/nest/nest-simulator/issues"
    echo "***"
    echo "*** To help us diagnose the problem, please attach the archived content"
    echo "*** of these directories to the issue:"
    echo "***     - '${TEST_OUTDIR}'"
    echo "***     - '${TEST_TMPDIR}'"
    echo "***"
    echo
}

#
# portable_inplace_sed file_name expression
#
portable_inplace_sed ()
{
    cp -f "$1" "$1.XXX"
    sed -e "$2" "$1.XXX" > "$1"
    rm -f "$1.XXX"
}

#
# measure runtime of command
#
time_cmd()
{
    start=`date +%s%N`
    $1
    end=`date +%s%N`
    echo `awk "BEGIN {print ($end - $start) / 1000000000}"`
}

#
# sed has different syntax for extended regular expressions
# on different operating systems:
# BSD: -E
# other: -r
#
EXTENDED_REGEX_PARAM=r
/bin/sh -c "echo 'hello' | sed -${EXTENDED_REGEX_PARAM} 's/[aeou]/_/g' "  >/dev/null 2>&1 || EXTENDED_REGEX_PARAM=E

JUNIT_FILE=
JUNIT_TESTS=
JUNIT_FAILURES=
JUNIT_CLASSNAME='core'

#
# junit_open file_name
#
junit_open ()
{
    if test "x$1" = x ; then
        bail_out 'junit_open: file_name not given!'
    fi

    JUNIT_FILE="${TEST_OUTDIR}/TEST-$1.xml"
    JUNIT_TESTS=0
    JUNIT_FAILURES=0

    TIME_TOTAL=0

    # Be compatible with BSD date; no --rfc-3339 and :z modifier
    timestamp="$( date -u '+%FT%T+00:00' )"

    echo '<?xml version="1.0" encoding="UTF-8" ?>' > "${JUNIT_FILE}"

    echo "<testsuite errors=\"0\" failures=XXX hostname=\"${INFO_HOST}\" name=\"$1\" tests=XXX time=XXX timestamp=\"${timestamp}\">" >> "${JUNIT_FILE}"
    echo '  <properties>' >> "${JUNIT_FILE}"
    echo "    <property name=\"os.arch\" value=\"${INFO_ARCH}\" />" >> "${JUNIT_FILE}"
    echo "    <property name=\"os.name\" value=\"${INFO_OS}\" />" >> "${JUNIT_FILE}"
    echo "    <property name=\"os.version\" value=\"${INFO_VER}\" />" >> "${JUNIT_FILE}"
    echo "    <property name=\"user.home\" value=\"${INFO_HOME}\" />" >> "${JUNIT_FILE}"
    echo "    <property name=\"user.name\" value=\"${INFO_USER}\" />" >> "${JUNIT_FILE}"
    echo '  </properties>' >> "${JUNIT_FILE}"
}

#
# junit_write classname testname [fail_message fail_trace]
#
junit_write ()
{
    if test "x${JUNIT_FILE}" = x ; then
        bail_out 'junit_write: report file not open, call junit_open first!'
    fi

    if test "x$1" = x || test "x$2" = x ; then
        bail_out 'junit_write: classname and testname arguments are mandatory!'
    fi

    JUNIT_TESTS=$(( ${JUNIT_TESTS} + 1 ))

    printf '%s' "  <testcase classname=\"$1\" name=\"$2\" time=\"${TIME_ELAPSED}\"" >> "${JUNIT_FILE}"

    if test "x$3" != x ; then
        echo '>' >> "${JUNIT_FILE}"
        echo "    <failure message=\"$3\" type=\"\"><![CDATA[" >> "${JUNIT_FILE}"
        echo "$4" | sed 's/]]>/]]>]]\&gt;<![CDATA[/' >> "${JUNIT_FILE}"
        echo "]]></failure>" >> "${JUNIT_FILE}"
        echo "  </testcase>" >> "${JUNIT_FILE}"
    else
        echo ' />' >> "${JUNIT_FILE}"
    fi
}

#
# junit_close
#
junit_close ()
{
    if test "x${JUNIT_FILE}" = x ; then
        bail_out 'junit_close: report file not open, call junit_open first!'
    fi

    portable_inplace_sed "${JUNIT_FILE}" "s/time=XXX/time=\"${TIME_TOTAL}\"/"
    portable_inplace_sed "${JUNIT_FILE}" "s/tests=XXX/tests=\"${JUNIT_TESTS}\"/"
    portable_inplace_sed "${JUNIT_FILE}" "s/failures=XXX/failures=\"${JUNIT_FAILURES}\"/"

    echo '  <system-out><![CDATA[]]></system-out>' >> "${JUNIT_FILE}"
    echo '  <system-err><![CDATA[]]></system-err>' >> "${JUNIT_FILE}"
    echo '</testsuite>' >> "${JUNIT_FILE}"

    JUNIT_FILE=
}

#
# run_test script_name codes_success codes_failure
#
# script_name: name of a .sli / .py script in $TEST_BASEDIR
#
# codes_success: variable that contains an explanation string for
#                all exit codes that are to be regarded as a success
# codes_skipped: variable that contains an explanation string for
#                all exit codes that mean the test was skipped
# codes_failure: variable that contains an explanation string for
#                all exit codes that are to be regarded as a failure
# Examples:
#
#   codes_success=' 0 Success'
#   codes_skipped=\
#   ' 200 Skipped,'\
#   ' 201 Skipped (MPI required),'\
#   ' 202 Skipped (build with-mpi=OFF required),'\
#   ' 203 Skipped (Threading required),'\
#   ' 204 Skipped (GSL required),'\
#   ' 205 Skipped (MUSIC required),'
#   codes_failure=\
#   ' 1 Failed: missed assertion,'\
#   ' 2 Failed: error in tested code block,'\
#   ' 3 Failed: tested code block failed to fail,'\
#   ' 126 Failed: error in test script,'
#
# Description:
#
#   The function runs the NEST binary with the SLI script script_name.
#   The exit code is then transformed into a human readable string
#   (if possible) using the global variables CODES_SUCCESS, CODES_SKIPPED, and
#   CODES_FAILURE which contain a comma separated list of exit codes
#   and strings describing the exit code.
#
#   If any of the variables CODES_SUCCESS, CODES_SKIPPED or CODES_FAILURE
#   contain an entry for the exit code, the respective string is logged, 
#   and the test is either counted as passed or failed.
#
#   If none of the variables CODES_SUCCESS, CODES_SKIPPED or CODES_FAILURE
#   contain an entry != "" for the returned exit code, the pass is counted as
#   failed, too, and unexpected exit code is logged).
#
# First Version by Ruediger Kupper, 07 Jul 2008
# Modified by Jochen Eppler and Markus Diesmann
# Modified by Yury V. Zaytsev
#
run_test ()
{
    TEST_TOTAL=$(( ${TEST_TOTAL} + 1 ))

    param_script="$1"
    param_success="$2"
    param_skipped="$3"
    param_failure="$4"

    msg_error=

    junit_class="$( echo "${param_script}" | sed 's/.[^.]\+$//' | sed 's/\/[^/]\+$//' | sed 's/\//./' )"
    junit_name="$( echo "${param_script}" | sed 's/^.*\/\([^/]\+\)$/\1/' )"

    echo          "Running test '${param_script}'... " >> "${TEST_LOGFILE}"
    printf '%s' "  Running test '${param_script}'... "

    # Very unfortunately, it is cheaper to generate a test runner on fly
    # rather than trying to fight with sh, dash, bash, etc. variable
    # expansion algorithms depending on whether the command is a built-in
    # or not, how many subshells have been forked and so on.

    echo "#!/bin/sh" >  "${TEST_RUNFILE}"
    echo "set +e"   >> "${TEST_RUNFILE}"

    echo "${param_script}" | grep -q '\.sli'
    if test $? -eq 0 ; then
      command="'${NEST_BINARY}' '${TEST_BASEDIR}/${param_script}' > '${TEST_OUTFILE}' 2>&1"
    else
      command="'${PYTHON}' '${TEST_BASEDIR}/${param_script}' > '${TEST_OUTFILE}' 2>&1"
    fi

    echo "${command}" >> "${TEST_RUNFILE}"
    echo "echo \$? > '${TEST_RETFILE}' ; exit 0" >> "${TEST_RUNFILE}"

    chmod 755 "${TEST_RUNFILE}"

    TIME_ELAPSED="$( time_cmd "${TEST_RUNFILE}" 2>&1 )"
    TIME_TOTAL="$( awk "BEGIN { print (${TIME_TOTAL} + ${TIME_ELAPSED}) ; }" )"
    rm -f "${TEST_RUNFILE}"

    exit_code="$(cat "${TEST_RETFILE}")"

    sed 's/^/   > /g' "${TEST_OUTFILE}" >> "${TEST_LOGFILE}"

    msg_dirty=${param_success##* ${exit_code} }
    msg_dirty_skip=${param_skipped##* ${exit_code} }
    msg_clean=${msg_dirty%%,*}
    if test "${msg_dirty}" != "${param_success}" ; then
        TEST_PASSED=$(( ${TEST_PASSED} + 1 ))
        explanation="${msg_clean}"
        junit_failure=
    elif test "${msg_dirty_skip}" != "${param_skipped}" ; then
        TEST_SKIPPED=$(( ${TEST_SKIPPED} + 1 ))
        msg_dirty=${msg_dirty_skip}
        msg_clean=${msg_dirty%%,*}
        explanation="${msg_clean}"
        junit_failure=
    else
        TEST_FAILED=$(( ${TEST_FAILED} + 1 ))
        JUNIT_FAILURES=$(( ${JUNIT_FAILURES} + 1 ))

	cat ${TEST_OUTFILE}

        msg_dirty=${param_failure##* ${exit_code} }
        msg_clean=${msg_dirty%%,*}
        msg_error="$( cat "${TEST_OUTFILE}" )"
        if test "${msg_dirty}" != "${param_failure}" ; then
            explanation="${msg_clean}"
        else
            explanation="Failed: unexpected exit code ${exit_code}"
            unexpected_exitcode=true
        fi

        junit_failure="${exit_code} (${explanation})"
    fi

    echo "${explanation}"

    if test "x${msg_error}" != x ; then
	echo ==================================================
	echo "Following is the full output of the test:"
	echo ==================================================
        echo "${msg_error}"
	echo ==================================================
    fi

    echo >> "${TEST_LOGFILE}" "-> ${exit_code} (${explanation})"
    echo >> "${TEST_LOGFILE}" "----------------------------------------"

    junit_write "${JUNIT_CLASSNAME}.${junit_class}" "${junit_name}" "${junit_failure}" "$(cat "${TEST_OUTFILE}")"

    # Panic on "unexpected" exit code
    if test "x${unexpected_exitcode}" != x ; then
        echo "***"
        echo "*** An unexpected exit code usually hints at a bug in the test suite!"
        ask_results
        exit 2
    fi

    rm -f "${TEST_OUTFILE}" "${TEST_RETFILE}"
}

# Set environment variables.
# The NEST_ variants of the global variables were set by
# cmake during configuration.
export PYTHONPATH=$NEST_PYTHONPATH:$PYTHONPATH
export PATH=$NEST_PATH:$PATH

# Gather some information about the host
INFO_ARCH="$(uname -m)"
INFO_HOME="$(/bin/sh -c 'echo ~')"
INFO_HOST="$(hostname -f)"
INFO_OS="$(uname -s)"
INFO_USER="$(whoami)"
INFO_VER="$(uname -r)"

TEST_BASEDIR=${NEST_DOC_DIR:-"@CMAKE_INSTALL_FULL_DOCDIR@"}
TEST_OUTDIR=${TEST_OUTDIR:-"$( pwd )/reports"}
TEST_LOGFILE="${TEST_OUTDIR}/installcheck.log"
TEST_OUTFILE="${TEST_OUTDIR}/output.log"
TEST_RETFILE="${TEST_OUTDIR}/output.ret"
TEST_RUNFILE="${TEST_OUTDIR}/runtest.sh"

if test -d "${TEST_OUTDIR}" ; then
    rm -rf "${TEST_OUTDIR}"
fi

mkdir "${TEST_OUTDIR}"

PYTHON="${PYTHON:-python}"
PYTHON_HARNESS="${NEST_DATA_DIR:-@CMAKE_INSTALL_FULL_DATADIR@}/extras/do_tests.py"

TMPDIR=${TMPDIR:-${TEST_OUTDIR}}
TEST_TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/nest.XXXXX")"
NEST_DATA_PATH="${TEST_TMPDIR}"
export NEST_DATA_PATH


# Check for old version of the /mpirun command, which had the NEST executable hardcoded
if test "x$(sli -c '/mpirun load cva_t Flatten length 3 eq =')" = xtrue ; then
    echo "  Unable to run tests because you compiled with MPI and ~/.nestrc contains"
    echo "  an old definition of /mpirun. If you were using the standard definition,"
    echo "  please replace it by"
    echo
    echo "  /mpirun"
    echo "  [/integertype /stringtype /stringtype]"
    echo "  [/numproc     /executable /scriptfile]"
    echo "  {"
    echo "   () ["
    echo "    (mpirun -np ) numproc cvs ( ) executable ( ) scriptfile"
    echo "   ] {join} Fold"
    echo "  } Function def"
    echo
    echo "  If you used a custom definition, please adapt it so that the signature"
    echo "  of your version matches the one above (i.e. taking number of processes,"
    echo "  executable and scriptfile as arguments; the old one just took number of"
    echo "  processes and slifile, the executable \"nest\" was hard-coded)."
    echo
    echo
    exit 1
fi


# Remember: single line exports are unportable!

NEST_BINARY=nest_serial

# Under Mac OS X, suppress crash reporter dialogs. Restore old state at end.
if test "x${INFO_OS}" = xDarwin ; then
    TEST_CRSTATE="$( defaults read com.apple.CrashReporter DialogType )"
    defaults write com.apple.CrashReporter DialogType server
fi

TEST_TOTAL=0
TEST_PASSED=0
TEST_SKIPPED=0
TEST_FAILED=0

TIME_TOTAL=0
TIME_ELAPSED=0

echo >  "${TEST_LOGFILE}" "NEST v. @NEST_VERSION_STRING@ testsuite log"
echo >> "${TEST_LOGFILE}" "======================"
echo >> "${TEST_LOGFILE}" "Running tests from ${TEST_BASEDIR}"

CODES_SKIPPED=\
' 200 Skipped,'\
' 201 Skipped (MPI required),'\
' 202 Skipped (build with-mpi=OFF required),'\
' 203 Skipped (Threading required),'\
' 204 Skipped (GSL required),'\
' 205 Skipped (MUSIC required),'

echo
echo 'Phase 1: Testing if SLI can execute scripts and report errors'
echo '-------------------------------------------------------------'

junit_open 'core.phase_1'

CODES_SUCCESS=' 0 Success'
CODES_FAILURE=
for test_name in test_pass.sli test_goodhandler.sli test_lazyhandler.sli ; do
    run_test "selftests/${test_name}" "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"
done

CODES_SUCCESS=' 126 Success'
CODES_FAILURE=
for test_name in test_fail.sli test_stop.sli test_badhandler.sli ; do
    run_test "selftests/${test_name}" "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"
done

junit_close

# At this point, we are sure that
#
#  * NEST will return 0 after finishing a script
#  * NEST will return 126 when a script raises an unhandled error
#  * Error handling in stopped contexts works

echo
echo "Phase 2: Testing SLI's unittest library"
echo "---------------------------------------"

junit_open 'core.phase_2'

# assert_or_die uses pass_or_die, so pass_or_die should be tested first.

CODES_SUCCESS=' 2 Success'
CODES_FAILURE=' 126 Failed: error in test script'

run_test selftests/test_pass_or_die.sli "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"

CODES_SUCCESS=' 1 Success'
CODES_FAILURE=\
' 2 Failed: error in tested code block,'\
' 126 Failed: error in test script,'

run_test selftests/test_assert_or_die_b.sli "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"
run_test selftests/test_assert_or_die_p.sli "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"

CODES_SUCCESS=' 3 Success'
CODES_FAILURE=\
' 1 Failed: missed assertion,'\
' 2 Failed: error in tested code block,'\
' 126 Failed: error in test script,'

run_test selftests/test_fail_or_die.sli "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"

CODES_SUCCESS=' 3 Success'
CODES_FAILURE=\
' 1 Failed: missed assertion,'\
' 2 Failed: error in tested code block,'\
' 126 Failed: error in test script,'

run_test selftests/test_crash_or_die.sli "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"

CODES_SUCCESS=' 3 Success'
CODES_FAILURE=\
' 1 Failed: missed assertion,'\
' 2 Failed: error in tested code block,'\
' 126 Failed: error in test script,'

run_test selftests/test_failbutnocrash_or_die_crash.sli "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"
run_test selftests/test_failbutnocrash_or_die_pass.sli "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"

CODES_SUCCESS=' 3 Success'
CODES_FAILURE=\
' 1 Failed: missed assertion,'\
' 2 Failed: error in tested code block,'\
' 126 Failed: error in test script,'

run_test selftests/test_passorfailbutnocrash_or_die.sli "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"

junit_close

# At this point, we are sure that
#
#  * unittest::pass_or_die works
#  * unittest::assert_or_die works
#  * unittest::fail_or_die works
#  * unittest::crash_or_die works

# These are the default exit codes and their explanations
CODES_SUCCESS=' 0 Success'
CODES_FAILURE=\
' 1 Failed: missed SLI assertion,'\
' 2 Failed: error in tested code block,'\
' 3 Failed: tested code block failed to fail,'\
' 4 Failed: re-run serial,'\
' 10 Failed: unknown error,'\
' 20 Failed: inconsistent copyright header(s),'\
' 30 Failed: inconsistent Name definition(s)/declaration(s),'\
' 31 Failed: unused Name definition(s),'\
' 125 Failed: unknown C++ exception,'\
' 126 Failed: error in test script,'\
' 127 Failed: fatal error,'\
' 134 Failed: missed C++ assertion,'\
' 139 Failed: segmentation fault,'

echo
echo "Phase 3: Running NEST unit tests"
echo "--------------------------------"

junit_open 'core.phase_3'

for test_ext in sli py ; do
      for test_name in $(ls "${TEST_BASEDIR}/unittests/" | grep ".*\.${test_ext}\$") ; do
          run_test "unittests/${test_name}" "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"
      done
done

junit_close

echo
echo "Phase 4: Running regression tests"
echo "---------------------------------"

junit_open 'core.phase_4'

for test_ext in sli py ; do
    for test_name in $(ls "${TEST_BASEDIR}/regressiontests/" | grep ".*\.${test_ext}$") ; do
        run_test "regressiontests/${test_name}" "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"
    done
done

junit_close

echo
echo "Phase 5: Running MPI tests"
echo "--------------------------"
if test "x$(sli -c 'statusdict/have_mpi :: =')" = xtrue ; then
    junit_open 'core.phase_5'

    NEST_BINARY=nest_indirect
    for test_name in $(ls "${TEST_BASEDIR}/mpi_selftests/pass" | grep '.*\.sli$') ; do
        run_test "mpi_selftests/pass/${test_name}" "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"
    done

    # tests meant to fail
    SAVE_CODES_SUCCESS=${CODES_SUCCESS}
    SAVE_CODES_FAILURE=${CODES_FAILURE}
    CODES_SUCCESS=' 1 Success (expected failure)'
    CODES_FAILURE=' 0 Failed: Unittest failed to detect error.'
    for test_name in $(ls "${TEST_BASEDIR}/mpi_selftests/fail" | grep '.*\.sli$') ; do
        run_test "mpi_selftests/fail/${test_name}" "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"
    done
    CODES_SUCCESS=${SAVE_CODES_SUCCESS}
    CODES_FAILURE=${SAVE_CODES_FAILURE}

    for test_name in $(ls "${TEST_BASEDIR}/mpitests/" | grep '.*\.sli$') ; do
        run_test "mpitests/${test_name}" "${CODES_SUCCESS}" "${CODES_SKIPPED}" "${CODES_FAILURE}"
    done

    junit_close
else
  echo "  Not running MPI tests because NEST was compiled without support"
  echo "  for distributed computing. See the file README.md for details."
fi

#echo
#echo "Phase 6: Running MUSIC tests"
#echo "----------------------------"
#if test "x$(sli -c 'statusdict/have_music :: =')" = xtrue ; then
#    junit_open 'core.phase_6'
#
#    BASEDIR=$PWD
#    tmpdir=$(mktemp -d)
#
#    TESTDIR=${TEST_BASEDIR}/musictests/
#
#    for test_name in $(ls ${TESTDIR} | grep '.*\.music$') ; do
#        music_file=${TESTDIR}/${test_name}
#
#        # Collect the list of SLI files from the .music file.
#        sli_files=$(grep '\.sli' ${music_file} | sed -e "s#args=#${TESTDIR}#g")
#        sli_files=$(for f in ${sli_files}; do if test -f ${f}; then echo ${f}; fi; done)
#
#        # Check if there is an accompanying shell script for the test.
#        sh_file=${TESTDIR}/$(basename ${music_file} .music).sh
#        if test ! -f ${sh_file}; then unset sh_file; fi
#
#        # Calculate the total number of processes in the .music file.
#        np=$(($(sed -n 's/np=//p' ${music_file} | paste -sd'+' -)))
#        command=$(sli -c "${np} (@MUSIC_EXECUTABLE@) (${test_name}) mpirun =")
#
#        proc_txt="processes"
#        if test $np -eq 1; then proc_txt="process"; fi
#        echo          "Running test '${test_name}' with $np $proc_txt... " >> "${TEST_LOGFILE}"
#        printf '%s' "  Running test '${test_name}' with $np $proc_txt... "
#
#        # Copy everything to the tmpdir.
#        cp ${music_file} ${sh_file} ${sli_files} ${tmpdir}
#        cd ${tmpdir}
#
#        # Create the runner script
#        echo "#!/bin/sh" >  runner.sh
#        echo "set +e" >> runner.sh
#        echo "export NEST_DATA_PATH=${tmpdir}" >> runner.sh
#        echo "${command} > output.log 2>&1" >> runner.sh
#        if test -n "${sh_file}"; then
#            chmod 755 $(basename ${sh_file})
#            echo "./"$(basename ${sh_file}) >> runner.sh
#        fi
#        echo "echo \$? > exit_code ; exit 0" >> runner.sh
#
#        # Run the script and copy all output to the logfile.
#        chmod 755 runner.sh
#        ./runner.sh
#        sed -e 's/^/   > /g' output.log >> "${TEST_LOGFILE}"
#
#        # Retrieve the exit code. This is either the one of the mpirun
#        # call or of the accompanying shell script if present.
#        exit_code=$(cat exit_code)
#
#        rm ${tmpdir}/*
#        cd ${BASEDIR}
#
#        # If the name of the test contains 'failure', we expect it to
#        # fail and the test logic is inverted.
#        TEST_TOTAL=$(( ${TEST_TOTAL} + 1 ))
#        if test -z $(echo ${test_name} | grep failure); then
#            if test $exit_code -eq 0 ; then
#                echo "Success"
#                TEST_PASSED=$(( ${TEST_PASSED} + 1 ))
#            elif test $exit_code -ge 200 && $exit_code -le 215; then
#                echo "Skipped"
#                TEST_SKIPPED=$(( ${TEST_SKIPPED} + 1 ))
#            else
#                echo "Failure"
#                TEST_FAILED=$(( ${TEST_FAILED} + 1 ))
#            fi
#        else
#            if test $exit_code -ne 0 ; then
#                echo "Success (expected failure)"
#                TEST_PASSED=$(( ${TEST_PASSED} + 1 ))
#            elif test $exit_code -ge 200 && $exit_code -le 215; then
#                echo "Skipped"
#                TEST_SKIPPED=$(( ${TEST_SKIPPED} + 1 ))
#            else
#                echo "Failure (test failed to fail)"
#                TEST_FAILED=$(( ${TEST_FAILED} + 1 ))
#            fi
#        fi
#    done
#
#    rm -rf $tmpdir
#
#    junit_close
#else
#  echo "  Not running MUSIC tests because NEST was compiled without support"
#  echo "  for it. See the file README.md for details."
#fi

if test "x${TEST_PYNEST}" = xtrue ; then

    echo
    echo "Phase 7: Running PyNEST tests."
    echo "------------------------------"
    
    # If possible, we run using nosetests. To find out if nosetests work, 
    # we proceed in two steps:
    # 1. Check if nosetests is available
    # 2. Check that nosetests supports --with-xunit by running nosetests.
    #    We need to run nosetests on a directory without any Python test
    #    files, because if they failed that would be interpreted as lack
    #    of support for nosetests. We use the TEST_OUTDIR as Python-free
    #    dummy directory to search for tests.

    if command -v @NOSETESTS@ >/dev/null 2>&1 && @PYTHON@ @NOSETESTS@ --with-xunit --xunit-file=/dev/null --where=${TEST_OUTDIR} >/dev/null 2>&1; then

        echo
        echo "  Using nosetests."
        echo

        @PYTHON@ @NOSETESTS@ -v --with-xunit --xunit-file=${TEST_OUTDIR}/pynest_tests.xml ${NEST_PYTHON_PREFIX:-@CMAKE_INSTALL_PREFIX@/@PYEXECDIR@}/nest/tests 2>&1 \
            | tee -a "${TEST_LOGFILE}" | grep -i --line-buffered "\.\.\. ok\|fail\|skip\|error" | sed 's/^/  /'

        PYNEST_TEST_TOTAL="$(  tail -n 3 ${TEST_LOGFILE} | grep Ran | cut -d' ' -f2 )"
        PYNEST_TEST_SKIPPED="$(  tail -n 1 ${TEST_LOGFILE} | sed -$EXTENDED_REGEX_PARAM 's/.*SKIP=([0-9]+).*/\1/')"
        PYNEST_TEST_FAILURES="$( tail -n 3 ${TEST_LOGFILE} | grep \( | sed -$EXTENDED_REGEX_PARAM 's/^[a-zA-Z]+ \((.*)\)/\1/' | sed -$EXTENDED_REGEX_PARAM 's/.*failures=([0-9]+).*/\1/' )"
        PYNEST_TEST_ERRORS="$( tail -n 3 ${TEST_LOGFILE} | grep \( | sed -$EXTENDED_REGEX_PARAM 's/^[a-zA-Z]+ \((.*)\)/\1/' | sed -$EXTENDED_REGEX_PARAM 's/.*errors=([0-9]+).*/\1/' )"

        # check that PYNEST_TEST_FAILURES/PYNEST_TEST_ERRORS contain numbers
        if test ${PYNEST_TEST_FAILURES} -eq ${PYNEST_TEST_FAILURES} 2>/dev/null ; then
          # PYNEST_TEST_FAILURES is a valid number
          :
        else
          PYNEST_TEST_FAILURES=0
        fi
        if test ${PYNEST_TEST_ERRORS} -eq ${PYNEST_TEST_ERRORS} 2>/dev/null ; then
          # PYNEST_TEST_ERRORS is a valid number
          :
        else
          PYNEST_TEST_ERRORS=0
        fi
        if test ${PYNEST_TEST_SKIPPED} -eq ${PYNEST_TEST_SKIPPED} 2>/dev/null ; then
          # PYNEST_TEST_SKIPPED is a valid number
          :
        else
          PYNEST_TEST_SKIPPED=0
        fi
        PYNEST_TEST_FAILED=$(($PYNEST_TEST_ERRORS + $PYNEST_TEST_FAILURES))
        PYNEST_TEST_PASSED=$(($PYNEST_TEST_TOTAL - $PYNEST_TEST_SKIPPED - $PYNEST_TEST_FAILED))

    else

        echo
        echo "  Nosetests unavailable. Using fallback test harness."
        echo "  Fewer tests will be excuted."
        echo

        "${PYTHON}" "${PYTHON_HARNESS}" >> "${TEST_LOGFILE}"

        PYNEST_TEST_TOTAL="$(  cat pynest_test_numbers.log | cut -d' ' -f1 )"
        PYNEST_TEST_PASSED="$( cat pynest_test_numbers.log | cut -d' ' -f2 )"
        PYNEST_TEST_SKIPPED="$( cat pynest_test_numbers.log | cut -d' ' -f3 )"
        PYNEST_TEST_FAILED="$( cat pynest_test_numbers.log | cut -d' ' -f4 )"

        rm -f "pynest_test_numbers.log"

    fi

    echo
    echo "  PyNEST tests: ${PYNEST_TEST_TOTAL}"
    echo "     Passed: ${PYNEST_TEST_PASSED}"
    echo "     Skipped: ${PYNEST_TEST_SKIPPED}"
    echo "     Failed: ${PYNEST_TEST_FAILED}"

    PYNEST_TEST_SKIPPED_TEXT="(${PYNEST_TEST_SKIPPED} PyNEST)"
    PYNEST_TEST_FAILED_TEXT="(${PYNEST_TEST_FAILED} PyNEST)"

    TEST_TOTAL=$(( $TEST_TOTAL + $PYNEST_TEST_TOTAL ))
    TEST_PASSED=$(( $TEST_PASSED + $PYNEST_TEST_PASSED ))
    TEST_SKIPPED=$(( $TEST_SKIPPED + $PYNEST_TEST_SKIPPED ))
    TEST_FAILED=$(( $TEST_FAILED + $PYNEST_TEST_FAILED ))
else
    echo
    echo "Phase 7: Running PyNEST tests"
    echo "-----------------------------"
    echo "  Not running PyNEST tests because NEST was compiled without support"
    echo "  for Python. See the file README.md for details."
fi

echo
echo "Phase 8: Running C++ tests (experimental)"
echo "-----------------------------------------"

if command -v ${NEST_PATH}/run_all_cpptests > /dev/null 2>&1; then
  CPP_TEST_OUTPUT="$(${NEST_PATH}/run_all_cpptests 2>&1)"
  # TODO:
  # We should check the return code ($?) of run_all_cpptests here to see
  # if a fatal crash occured. We cannot simply test $? != 0, since
  # run_all_cpptests will return a non-zero code if tests fail.

  # TODO:
  # The regex for the total number seems fine according to
  # https://www.boost.org/doc/libs/1_65_0/libs/test/doc/html/boost_test/test_output/log_formats/log_human_readable_format.html
  # but does the check for failures work as it should?
  # At some point, we should also count skipped tests.
  CPP_TEST_TOTAL=$(echo "$CPP_TEST_OUTPUT" | sed -${EXTENDED_REGEX_PARAM} -n 's/.*Running ([0-9]+).*/\1/p')
  CPP_TEST_FAILED=$(echo "$CPP_TEST_OUTPUT" | sed -${EXTENDED_REGEX_PARAM} -n 's/.*([0-9]+) failure.*/\1/p')

  # replace empty strings by zero so arithmetic expressions below work
  CPP_TEST_TOTAL=${CPP_TEST_TOTAL:-0}
  CPP_TEST_FAILED=${CPP_TEST_FAILED:-0}
  CPP_TEST_PASSED=$(( $CPP_TEST_TOTAL - $CPP_TEST_FAILED ))

  TEST_TOTAL=$(( $TEST_TOTAL + $CPP_TEST_TOTAL ))
  TEST_PASSED=$(( $TEST_PASSED + $CPP_TEST_PASSED ))
  TEST_FAILED=$(( $TEST_FAILED + $CPP_TEST_FAILED ))

  echo "  C++ tests : ${CPP_TEST_TOTAL}"
  echo "     Passed : ${CPP_TEST_PASSED}"
  echo "     Failed : ${CPP_TEST_FAILED}"

else
  echo "  Not running C++ tests because NEST was compiled without Boost."
fi

echo
echo "NEST Testsuite Summary"
echo "----------------------"
echo "  NEST Executable: $(which nest)"
echo "  SLI Executable : $(which sli)"
echo "  Total number of tests: ${TEST_TOTAL}"
echo "     Passed: ${TEST_PASSED}"
echo "     Skipped: ${TEST_SKIPPED} ${PYNEST_TEST_SKIPPED_TEXT:-}"
echo "     Failed: ${TEST_FAILED} ${PYNEST_TEST_FAILED_TEXT:-}"
echo

if test ${TEST_FAILED} -gt 0 ; then
    echo "***"
    echo "*** There were errors detected during the run of the NEST test suite!"
    ask_results
else
    rm -rf "${TEST_TMPDIR}"
fi

# Mac OS X: Restore old crash reporter state
if test "x${INFO_OS}" = xDarwin ; then
    defaults write com.apple.CrashReporter DialogType "${TEST_CRSTATE}"
fi

if test ${TEST_FAILED} -gt 0 ; then
    exit 1
else
    exit 0
fi
